{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-03-21T12:08:51.063709Z",
     "iopub.status.busy": "2022-03-21T12:08:51.063258Z",
     "iopub.status.idle": "2022-03-21T12:08:59.305708Z",
     "shell.execute_reply": "2022-03-21T12:08:59.304877Z",
     "shell.execute_reply.started": "2022-03-21T12:08:51.063657Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import keras\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Input, Dense, LSTM, GRU, Embedding\n",
    "from keras.layers import Activation, Bidirectional, GlobalMaxPool1D, GlobalMaxPool2D, Dropout\n",
    "from keras.models import Model\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.optimizers import RMSprop, adam\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer,PorterStemmer\n",
    "import seaborn as sns\n",
    "import transformers\n",
    "from transformers import AutoTokenizer\n",
    "from tokenizers import BertWordPieceTokenizer\n",
    "from keras.initializers import Constant\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "from collections import Counter\n",
    "\n",
    "stop=set(stopwords.words('english'))\n",
    "\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:08:59.307911Z",
     "iopub.status.busy": "2022-03-21T12:08:59.30762Z",
     "iopub.status.idle": "2022-03-21T12:08:59.326826Z",
     "shell.execute_reply": "2022-03-21T12:08:59.325926Z",
     "shell.execute_reply.started": "2022-03-21T12:08:59.307879Z"
    }
   },
   "outputs": [],
   "source": [
    "#https://www.kaggle.com/shahules/complete-eda-baseline-model-0-708-lb\n",
    "\n",
    "def basic_cleaning(text):\n",
    "    text=re.sub(r'https?://www\\.\\S+\\.com','',text)\n",
    "    text=re.sub(r'[^A-Za-z|\\s]','',text)\n",
    "    text=re.sub(r'\\*+','swear',text) #capture swear words that are **** out\n",
    "    return text\n",
    "\n",
    "def remove_html(text):\n",
    "    html=re.compile(r'<.*?>')\n",
    "    return html.sub(r'',text)\n",
    "\n",
    "# Reference : https://gist.github.com/slowkow/7a7f61f495e3dbb7e3d767f97bd7304b\n",
    "def remove_emoji(text):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           u\"\\U00002702-\\U000027B0\"\n",
    "                           u\"\\U000024C2-\\U0001F251\"\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', text)\n",
    "\n",
    "def remove_multiplechars(text):\n",
    "    text = re.sub(r'(.)\\1{3,}',r'\\1', text)\n",
    "    return text\n",
    "\n",
    "\n",
    "def clean(df):\n",
    "    for col in ['text']:#,'selected_text']:\n",
    "        df[col]=df[col].astype(str).apply(lambda x:basic_cleaning(x))\n",
    "        df[col]=df[col].astype(str).apply(lambda x:remove_emoji(x))\n",
    "        df[col]=df[col].astype(str).apply(lambda x:remove_html(x))\n",
    "        df[col]=df[col].astype(str).apply(lambda x:remove_multiplechars(x))\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:08:59.328953Z",
     "iopub.status.busy": "2022-03-21T12:08:59.328416Z",
     "iopub.status.idle": "2022-03-21T12:08:59.338407Z",
     "shell.execute_reply": "2022-03-21T12:08:59.337464Z",
     "shell.execute_reply.started": "2022-03-21T12:08:59.328918Z"
    }
   },
   "outputs": [],
   "source": [
    "def fast_encode(texts, tokenizer, chunk_size=256, maxlen=128):    \n",
    "    tokenizer.enable_truncation(max_length=maxlen)\n",
    "    tokenizer.enable_padding(max_length=maxlen)\n",
    "    all_ids = []\n",
    "    \n",
    "    for i in range(0, len(texts), chunk_size):\n",
    "        text_chunk = texts[i:i+chunk_size].tolist()\n",
    "        encs = tokenizer.encode_batch(text_chunk)\n",
    "        all_ids.extend([enc.ids for enc in encs])\n",
    "    \n",
    "    return np.array(all_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:08:59.339977Z",
     "iopub.status.busy": "2022-03-21T12:08:59.339659Z",
     "iopub.status.idle": "2022-03-21T12:08:59.348864Z",
     "shell.execute_reply": "2022-03-21T12:08:59.348119Z",
     "shell.execute_reply.started": "2022-03-21T12:08:59.339942Z"
    }
   },
   "outputs": [],
   "source": [
    "def preprocess_news(df,stop=stop,n=1,col='text'):\n",
    "    '''Function to preprocess and create corpus'''\n",
    "    new_corpus=[]\n",
    "    stem=PorterStemmer()\n",
    "    lem=WordNetLemmatizer()\n",
    "    for text in df[col]:\n",
    "        words=[w for w in word_tokenize(text) if (w not in stop)]\n",
    "       \n",
    "        words=[lem.lemmatize(w) for w in words if(len(w)>n)]\n",
    "     \n",
    "        new_corpus.append(words)\n",
    "        \n",
    "    new_corpus=[word for l in new_corpus for word in l]\n",
    "    return new_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2022-03-21T12:08:59.352729Z",
     "iopub.status.busy": "2022-03-21T12:08:59.351891Z",
     "iopub.status.idle": "2022-03-21T12:08:59.493439Z",
     "shell.execute_reply": "2022-03-21T12:08:59.492817Z",
     "shell.execute_reply.started": "2022-03-21T12:08:59.352694Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>textID</th>\n",
       "      <th>text</th>\n",
       "      <th>selected_text</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cb774db0d1</td>\n",
       "      <td>I`d have responded, if I were going</td>\n",
       "      <td>I`d have responded, if I were going</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>549e992a42</td>\n",
       "      <td>Sooo SAD I will miss you here in San Diego!!!</td>\n",
       "      <td>Sooo SAD</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>088c60f138</td>\n",
       "      <td>my boss is bullying me...</td>\n",
       "      <td>bullying me</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9642c003ef</td>\n",
       "      <td>what interview! leave me alone</td>\n",
       "      <td>leave me alone</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>358bd9e861</td>\n",
       "      <td>Sons of ****, why couldn`t they put them on t...</td>\n",
       "      <td>Sons of ****,</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       textID                                               text  \\\n",
       "0  cb774db0d1                I`d have responded, if I were going   \n",
       "1  549e992a42      Sooo SAD I will miss you here in San Diego!!!   \n",
       "2  088c60f138                          my boss is bullying me...   \n",
       "3  9642c003ef                     what interview! leave me alone   \n",
       "4  358bd9e861   Sons of ****, why couldn`t they put them on t...   \n",
       "\n",
       "                         selected_text sentiment  \n",
       "0  I`d have responded, if I were going   neutral  \n",
       "1                             Sooo SAD  negative  \n",
       "2                          bullying me  negative  \n",
       "3                       leave me alone  negative  \n",
       "4                        Sons of ****,  negative  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "We drop the NaN values and have 4 helper functions to clean the data. These are:\n",
    "1. basic_cleaning - to remove website urls, non-characters and to replace '*****' swear words with the word swear\n",
    "2. remove_html\n",
    "3. remove_emojis\n",
    "4. remove_multiplechars - this is for when there are more than 3 characters in a row in a word e.g. wayyyyy. The function removes all but one of the letters\n",
    "\n",
    "The data is then ready for initial exploration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:08:59.496995Z",
     "iopub.status.busy": "2022-03-21T12:08:59.49678Z",
     "iopub.status.idle": "2022-03-21T12:09:00.183334Z",
     "shell.execute_reply": "2022-03-21T12:09:00.182625Z",
     "shell.execute_reply.started": "2022-03-21T12:08:59.496971Z"
    }
   },
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)\n",
    "\n",
    "df_clean = clean(df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the labels, one-hot encoding performed significantly better than LabelEncoder. We also tokenize and covert to sequences. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:09:00.185347Z",
     "iopub.status.busy": "2022-03-21T12:09:00.185032Z",
     "iopub.status.idle": "2022-03-21T12:09:00.202797Z",
     "shell.execute_reply": "2022-03-21T12:09:00.202192Z",
     "shell.execute_reply.started": "2022-03-21T12:09:00.185313Z"
    }
   },
   "outputs": [],
   "source": [
    "df_clean_selection = df_clean.sample(frac=1)\n",
    "X = df_clean_selection.text.values\n",
    "y = pd.get_dummies(df_clean_selection.sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:09:00.204355Z",
     "iopub.status.busy": "2022-03-21T12:09:00.204085Z",
     "iopub.status.idle": "2022-03-21T12:09:01.563424Z",
     "shell.execute_reply": "2022-03-21T12:09:01.562734Z",
     "shell.execute_reply.started": "2022-03-21T12:09:00.204321Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = text.Tokenizer(num_words=20000)\n",
    "tokenizer.fit_on_texts(list(X))\n",
    "list_tokenized_train = tokenizer.texts_to_sequences(X)\n",
    "X_t = sequence.pad_sequences(list_tokenized_train, maxlen=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The model: DistilBert"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now turn our attention to pretrained embeddings. In this case we download and use DistilBert instead of training our own Embedding layer. DistilBert, a light version of BERT, google's game-changing NLP model, provides us with a tokenizer and an embedding matrix. BERT base uncased is trained on lower case English text and has around 110 million parameters (768 dimensions for embedding each word, and a vocab of 143,000 words). Distilbert has 60% of this, but maintains 97% performance against BERT. \n",
    "\n",
    "For the purposes of this example, we will leave that matrix rather than train it, as it's large and we would have unrealistic training times. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:09:01.564985Z",
     "iopub.status.busy": "2022-03-21T12:09:01.564708Z",
     "iopub.status.idle": "2022-03-21T12:09:03.955881Z",
     "shell.execute_reply": "2022-03-21T12:09:03.955144Z",
     "shell.execute_reply.started": "2022-03-21T12:09:01.56495Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b350e80ad8a448b864c72817a89437a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69a50fffddf7455e830aa06e9963e514",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Tokenizer(vocabulary_size=30522, model=BertWordPiece, add_special_tokens=True, unk_token=[UNK], sep_token=[SEP], cls_token=[CLS], clean_text=True, handle_chinese_chars=True, strip_accents=True, lowercase=True, wordpieces_prefix=##)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = transformers.AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")  ## change it to commit\n",
    "\n",
    "# Save the loaded tokenizer locally\n",
    "save_path = '/kaggle/working/distilbert_base_uncased/'\n",
    "if not os.path.exists(save_path):\n",
    "    os.makedirs(save_path)\n",
    "tokenizer.save_pretrained(save_path)\n",
    "\n",
    "# Reload it with the huggingface tokenizers library\n",
    "fast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=True)\n",
    "fast_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:09:03.957565Z",
     "iopub.status.busy": "2022-03-21T12:09:03.957281Z",
     "iopub.status.idle": "2022-03-21T12:09:06.298544Z",
     "shell.execute_reply": "2022-03-21T12:09:06.29789Z",
     "shell.execute_reply.started": "2022-03-21T12:09:03.957528Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27480, 128)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = fast_encode(df_clean_selection.text.astype(str), fast_tokenizer, maxlen=128)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:09:06.300204Z",
     "iopub.status.busy": "2022-03-21T12:09:06.299927Z",
     "iopub.status.idle": "2022-03-21T12:09:27.640166Z",
     "shell.execute_reply": "2022-03-21T12:09:27.639415Z",
     "shell.execute_reply.started": "2022-03-21T12:09:06.300168Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfd9bd15b65b4b14b0885699611c9cbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=363423424.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "transformer_layer = transformers.TFDistilBertModel.from_pretrained('distilbert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:09:27.641943Z",
     "iopub.status.busy": "2022-03-21T12:09:27.641654Z",
     "iopub.status.idle": "2022-03-21T12:09:28.29538Z",
     "shell.execute_reply": "2022-03-21T12:09:28.294529Z",
     "shell.execute_reply.started": "2022-03-21T12:09:27.64191Z"
    }
   },
   "outputs": [],
   "source": [
    "embedding_size = 128\n",
    "input_ = Input(shape=(100,))\n",
    "\n",
    "inp = Input(shape=(128, ))\n",
    "#inp2= Input(shape=(1,))\n",
    "\n",
    "embedding_matrix=transformer_layer.weights[0].numpy()\n",
    "\n",
    "x = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],embeddings_initializer=Constant(embedding_matrix),trainable=False)(inp)\n",
    "x = Bidirectional(LSTM(50, return_sequences=True))(x)\n",
    "x = Bidirectional(LSTM(25, return_sequences=True))(x)\n",
    "x = GlobalMaxPool1D()(x)\n",
    "x = Dropout(0.5)(x)\n",
    "x = Dense(50, activation='relu', kernel_regularizer='L1L2')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "x = Dense(3, activation='softmax')(x)\n",
    "\n",
    "model_DistilBert = Model(inputs=[inp], outputs=x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:09:28.297174Z",
     "iopub.status.busy": "2022-03-21T12:09:28.296891Z",
     "iopub.status.idle": "2022-03-21T12:09:28.858446Z",
     "shell.execute_reply": "2022-03-21T12:09:28.857763Z",
     "shell.execute_reply.started": "2022-03-21T12:09:28.297141Z"
    }
   },
   "outputs": [],
   "source": [
    "model_DistilBert.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:09:28.859932Z",
     "iopub.status.busy": "2022-03-21T12:09:28.859651Z",
     "iopub.status.idle": "2022-03-21T12:09:28.87197Z",
     "shell.execute_reply": "2022-03-21T12:09:28.871231Z",
     "shell.execute_reply.started": "2022-03-21T12:09:28.859901Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 128, 768)          23440896  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 128, 100)          327600    \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 128, 50)           25200     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 3)                 153       \n",
      "=================================================================\n",
      "Total params: 23,796,399\n",
      "Trainable params: 355,503\n",
      "Non-trainable params: 23,440,896\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_DistilBert.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:11:42.385184Z",
     "iopub.status.busy": "2022-03-21T12:11:42.384903Z",
     "iopub.status.idle": "2022-03-21T12:17:03.044708Z",
     "shell.execute_reply": "2022-03-21T12:17:03.043802Z",
     "shell.execute_reply.started": "2022-03-21T12:11:42.385156Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 24732 samples, validate on 2748 samples\n",
      "Epoch 1/1\n",
      "24732/24732 [==============================] - 325s 13ms/step - loss: 1.0742 - accuracy: 0.4145 - val_loss: 0.9324 - val_accuracy: 0.5368\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f9094031f98>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model_DistilBert.fit(X,y,batch_size=32,epochs=10,validation_split=0.1) \n",
    "model_DistilBert.fit(X,y,batch_size=32,epochs=1,validation_split=0.1) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean_final = df_clean.sample(frac=1)\n",
    "X_train = fast_encode(df_clean_selection.text.astype(str), fast_tokenizer, maxlen=128)\n",
    "y_train = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "27480/27480 [==============================] - 438s 16ms/step - loss: 0.7857 - accuracy: 0.6560\n",
      "Epoch 2/10\n",
      "27480/27480 [==============================] - 436s 16ms/step - loss: 0.6661 - accuracy: 0.7261\n",
      "Epoch 3/10\n",
      "27480/27480 [==============================] - 435s 16ms/step - loss: 0.6358 - accuracy: 0.7460\n",
      "Epoch 4/10\n",
      "27480/27480 [==============================] - 437s 16ms/step - loss: 0.6108 - accuracy: 0.7564\n",
      "Epoch 5/10\n",
      "27480/27480 [==============================] - 436s 16ms/step - loss: 0.5887 - accuracy: 0.7656\n",
      "Epoch 6/10\n",
      "27480/27480 [==============================] - 435s 16ms/step - loss: 0.5710 - accuracy: 0.7738\n",
      "Epoch 7/10\n",
      "27480/27480 [==============================] - 436s 16ms/step - loss: 0.5480 - accuracy: 0.7826\n",
      "Epoch 8/10\n",
      "27480/27480 [==============================] - 435s 16ms/step - loss: 0.5333 - accuracy: 0.7902\n",
      "Epoch 9/10\n",
      "27480/27480 [==============================] - 436s 16ms/step - loss: 0.5138 - accuracy: 0.7977\n",
      "Epoch 10/10\n",
      "27480/27480 [==============================] - 436s 16ms/step - loss: 0.4953 - accuracy: 0.8063\n"
     ]
    }
   ],
   "source": [
    "Adam_name = adam(lr=0.001)\n",
    "model_DistilBert.compile(loss='categorical_crossentropy',optimizer=Adam_name,metrics=['accuracy'])\n",
    "history = model_DistilBert.fit(X_train,y_train,batch_size=32,epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:18:30.87925Z",
     "iopub.status.busy": "2022-03-21T12:18:30.878971Z",
     "iopub.status.idle": "2022-03-21T12:18:31.268527Z",
     "shell.execute_reply": "2022-03-21T12:18:31.267787Z",
     "shell.execute_reply.started": "2022-03-21T12:18:30.87922Z"
    }
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')\n",
    "df_test.dropna(inplace=True)\n",
    "df_clean_test = clean(df_test)\n",
    "\n",
    "X_test = fast_encode(df_clean_test.text.values.astype(str), fast_tokenizer, maxlen=128)\n",
    "y_test = df_clean_test.sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:23:15.473895Z",
     "iopub.status.busy": "2022-03-21T12:23:15.473589Z",
     "iopub.status.idle": "2022-03-21T12:23:23.432212Z",
     "shell.execute_reply": "2022-03-21T12:23:23.431412Z",
     "shell.execute_reply.started": "2022-03-21T12:23:15.473868Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The final model shows 0.75 accuracy on the test set.\n"
     ]
    }
   ],
   "source": [
    "y_preds = model_DistilBert.predict(X_test)\n",
    "y_predictions = pd.DataFrame(y_preds, columns=['negative','neutral','positive'])\n",
    "y_predictions_final = y_predictions.idxmax(axis=1)\n",
    "accuracy = accuracy_score(y_test,y_predictions_final)\n",
    "print(f\"The final model shows {accuracy:.2f} accuracy on the test set.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-03-21T12:23:23.434005Z",
     "iopub.status.busy": "2022-03-21T12:23:23.433768Z",
     "iopub.status.idle": "2022-03-21T12:23:23.451582Z",
     "shell.execute_reply": "2022-03-21T12:23:23.450789Z",
     "shell.execute_reply.started": "2022-03-21T12:23:23.433979Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>textID</th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>predicted_negative</th>\n",
       "      <th>predicted_neutral</th>\n",
       "      <th>predicted_positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f87dea47db</td>\n",
       "      <td>Last session of the day  httptwitpiccomezh</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.038187</td>\n",
       "      <td>0.955291</td>\n",
       "      <td>0.006521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>96d74cb729</td>\n",
       "      <td>Shanghai is also really exciting precisely  s...</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.000121</td>\n",
       "      <td>0.028929</td>\n",
       "      <td>0.970950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eee518ae67</td>\n",
       "      <td>Recession hit Veronique Branquinho she has to ...</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.996926</td>\n",
       "      <td>0.003065</td>\n",
       "      <td>0.000009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01082688c6</td>\n",
       "      <td>happy bday</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.000117</td>\n",
       "      <td>0.021627</td>\n",
       "      <td>0.978256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33987a8ee5</td>\n",
       "      <td>httptwitpiccomwp  I like it</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.004728</td>\n",
       "      <td>0.156326</td>\n",
       "      <td>0.838946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>726e501993</td>\n",
       "      <td>thats great weee visitors</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.000108</td>\n",
       "      <td>0.025948</td>\n",
       "      <td>0.973944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>261932614e</td>\n",
       "      <td>I THINK EVERYONE HATES ME ON HERE   lol</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.940344</td>\n",
       "      <td>0.058028</td>\n",
       "      <td>0.001628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>afa11da83f</td>\n",
       "      <td>so wish i could but im in school and myspace ...</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.938790</td>\n",
       "      <td>0.059335</td>\n",
       "      <td>0.001876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>e64208b4ef</td>\n",
       "      <td>and within a short time of the last clue all ...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.096403</td>\n",
       "      <td>0.885329</td>\n",
       "      <td>0.018269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>37bcad24ca</td>\n",
       "      <td>What did you get  My day is alright havent do...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.280386</td>\n",
       "      <td>0.683009</td>\n",
       "      <td>0.036605</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       textID                                               text sentiment  \\\n",
       "0  f87dea47db         Last session of the day  httptwitpiccomezh   neutral   \n",
       "1  96d74cb729   Shanghai is also really exciting precisely  s...  positive   \n",
       "2  eee518ae67  Recession hit Veronique Branquinho she has to ...  negative   \n",
       "3  01082688c6                                         happy bday  positive   \n",
       "4  33987a8ee5                        httptwitpiccomwp  I like it  positive   \n",
       "5  726e501993                          thats great weee visitors  positive   \n",
       "6  261932614e            I THINK EVERYONE HATES ME ON HERE   lol  negative   \n",
       "7  afa11da83f   so wish i could but im in school and myspace ...  negative   \n",
       "8  e64208b4ef   and within a short time of the last clue all ...   neutral   \n",
       "9  37bcad24ca   What did you get  My day is alright havent do...   neutral   \n",
       "\n",
       "   predicted_negative  predicted_neutral  predicted_positive  \n",
       "0            0.038187           0.955291            0.006521  \n",
       "1            0.000121           0.028929            0.970950  \n",
       "2            0.996926           0.003065            0.000009  \n",
       "3            0.000117           0.021627            0.978256  \n",
       "4            0.004728           0.156326            0.838946  \n",
       "5            0.000108           0.025948            0.973944  \n",
       "6            0.940344           0.058028            0.001628  \n",
       "7            0.938790           0.059335            0.001876  \n",
       "8            0.096403           0.885329            0.018269  \n",
       "9            0.280386           0.683009            0.036605  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display = df_clean_test.copy()\n",
    "y_predictions.columns = ['predicted_' + f for f in y_predictions.columns]\n",
    "pd.concat([display, y_predictions], axis = 1).head(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "037ace54361649edb8a32acaa900a258": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "12d95c0044c8487fb3d773cba424b303": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "48f1f27028c344a98a3e92f27d212547": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "541a7017333d4afc918bd5be113bdb50": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5b350e80ad8a448b864c72817a89437a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_962389ad7dc2473fad406e39c30bf482",
        "IPY_MODEL_b5ec0dbce26d4d17874515c3835f512a"
       ],
       "layout": "IPY_MODEL_48f1f27028c344a98a3e92f27d212547"
      }
     },
     "5c7eb5f28ec04057a80310e577cc13b8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7e7233d032e549718a408edb2a836239",
       "max": 363423424.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_af2e9a3585c742f3a641c6b29e0bb1fd",
       "value": 363423424.0
      }
     },
     "6167234cfaa043cc9e7d489141dc31f6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "69a50fffddf7455e830aa06e9963e514": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_a19f67385ec14652bdf0f53ff0c7d266",
        "IPY_MODEL_f84d12ad59a84fb2b8f3aa36b1c270ab"
       ],
       "layout": "IPY_MODEL_a5665dea0d7b4e86bc20ec043ddbe60c"
      }
     },
     "711aa437a78f4af68bf44875ba6124d9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7e7233d032e549718a408edb2a836239": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "962389ad7dc2473fad406e39c30bf482": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_711aa437a78f4af68bf44875ba6124d9",
       "max": 442.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_f65ee2d9b58f4460a752c1b365bfce15",
       "value": 442.0
      }
     },
     "975a3eba5e1e42f3b7f53d1e8211f36a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c09bb56a9a7048f894f9340a1489a313",
       "placeholder": "​",
       "style": "IPY_MODEL_e778b61946b548948f152b1ef4bcd1c6",
       "value": " 363M/363M [06:07&lt;00:00, 990kB/s]"
      }
     },
     "a19f67385ec14652bdf0f53ff0c7d266": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_037ace54361649edb8a32acaa900a258",
       "max": 231508.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d307474196274a419225128b3e6470cc",
       "value": 231508.0
      }
     },
     "a5665dea0d7b4e86bc20ec043ddbe60c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "af2e9a3585c742f3a641c6b29e0bb1fd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "b5ec0dbce26d4d17874515c3835f512a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_12d95c0044c8487fb3d773cba424b303",
       "placeholder": "​",
       "style": "IPY_MODEL_cb0158ae479c4ec0a8c2bd20c73aa454",
       "value": " 442/442 [00:03&lt;00:00, 126B/s]"
      }
     },
     "c09bb56a9a7048f894f9340a1489a313": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cb0158ae479c4ec0a8c2bd20c73aa454": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "d307474196274a419225128b3e6470cc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "dfd9bd15b65b4b14b0885699611c9cbf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5c7eb5f28ec04057a80310e577cc13b8",
        "IPY_MODEL_975a3eba5e1e42f3b7f53d1e8211f36a"
       ],
       "layout": "IPY_MODEL_6167234cfaa043cc9e7d489141dc31f6"
      }
     },
     "e778b61946b548948f152b1ef4bcd1c6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ec254a4ffdd246ffaae5bbfc2cb95e51": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f65ee2d9b58f4460a752c1b365bfce15": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "f84d12ad59a84fb2b8f3aa36b1c270ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_541a7017333d4afc918bd5be113bdb50",
       "placeholder": "​",
       "style": "IPY_MODEL_ec254a4ffdd246ffaae5bbfc2cb95e51",
       "value": " 232k/232k [00:00&lt;00:00, 308kB/s]"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
